# This is a sample Python script.

# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
import time
import pymysql
import requests #导入调用网址的库
from bs4 import BeautifulSoup #导入解析库，一般用lxml
import re
import numpy as np
import pandas as pd
# from twisted.conch.scripts.conch import conn


# 爬取50部电影的200条评论
#直接写出来50部电影的详情页链接和名字

movie_urls =[
    'https://movie.douban.com/subject/1291546/','https://movie.douban.com/subject/1292720/','https://movie.douban.com/subject/1292722/','https://movie.douban.com/subject/1295644/','https://movie.douban.com/subject/1292063/',
    'https://movie.douban.com/subject/1291561/','https://movie.douban.com/subject/1295124/','https://movie.douban.com/subject/3541415/','https://movie.douban.com/subject/3011091/','https://movie.douban.com/subject/1889243/',
    'https://movie.douban.com/subject/1292064/', 'https://movie.douban.com/subject/1292001/', 'https://movie.douban.com/subject/3793023/', 'https://movie.douban.com/subject/2131459/', 'https://movie.douban.com/subject/1291549/',
    'https://movie.douban.com/subject/1307914/', 'https://movie.douban.com/subject/25662329/', 'https://movie.douban.com/subject/1292213/', 'https://movie.douban.com/subject/5912992/', 'https://movie.douban.com/subject/1291841/',
    'https://movie.douban.com/subject/1849031/', 'https://movie.douban.com/subject/1296141/', 'https://movie.douban.com/subject/3319755/', 'https://movie.douban.com/subject/1291560/', 'https://movie.douban.com/subject/6786002/',
    'https://movie.douban.com/subject/1293172/', 'https://movie.douban.com/subject/20495023/', 'https://movie.douban.com/subject/1851857/', 'https://movie.douban.com/subject/1292365/', 'https://movie.douban.com/subject/1295038/',
    'https://movie.douban.com/subject/1291552/', 'https://movie.douban.com/subject/1300267/', 'https://movie.douban.com/subject/21937452/', 'https://movie.douban.com/subject/2129039/', 'https://movie.douban.com/subject/2129039/',
    'https://movie.douban.com/subject/30170448/', 'https://movie.douban.com/subject/26752088/', 'https://movie.douban.com/subject/1293182/', 'https://movie.douban.com/subject/1308807/', 'https://movie.douban.com/subject/1929463/',
    'https://movie.douban.com/subject/1291858/', 'https://movie.douban.com/subject/1299398/', 'https://movie.douban.com/subject/1291583/', 'https://movie.douban.com/subject/1305487/', 'https://movie.douban.com/subject/1291828/',
    'https://movie.douban.com/subject/1291572/', 'https://movie.douban.com/subject/1298624/', 'https://movie.douban.com/subject/1293839/', 'https://movie.douban.com/subject/1296736/', 'https://movie.douban.com/subject/3742360/',
]

movie_name = ['霸王别姬', '阿甘正传', '泰坦尼克号', '这个杀手不太冷', '美丽人生', '千与千寻', '辛德勒的名单', '盗梦空间', '忠犬八公的故事', '星际穿越',
              '楚门的世界', '海上钢琴师', '三傻大闹宝莱坞', '机器人总动员', '放牛班的春天', '无间道', '疯狂动物城', '大话西游之大圣娶亲', '熔炉', '教父',
              '当幸福来敲门', '控方证人', '怦然心动', '龙猫', '触不可及', '末代皇帝', '寻梦环游记', '蝙蝠侠：黑暗骑士', '活着', '哈利波特与魔法石'
              '指环王3：王者无敌', '乱世佳人', '素媛', '飞屋环游记', '摔跤吧！爸爸', '何以为家', '我不是药神', '十二怒汉', '哈尔的移动城堡', '少年派的奇幻漂流',
              '鬼子来了', '大话西游之月光宝盒', '天空之城', '猫鼠游戏', '天堂电影院', '指环王2：双塔奇兵', '闻香识女人', '罗马假日', '钢琴家', '让子弹飞']

# 这个函数用于通过电影的链接和想要选取的影评的第i页获取对应第i页的影评链接
def get_com_url(movie_url,i): #这里i小于等于10页
    print("bbb")
    com_url = movie_url + 'comments?start={}&limit=20&status=F&sort=new_score'.format(i * 20) #里面status=P是看过，=F是想看
    return com_url

# 这里传入影评链接,i是为了在最后写入数据库的时候写入电影的名字
def get_com(com_url, i):
    print("aaa")
    com_url = com_url
    Cookie = {'Cookie':'douban-fav-remind=1; ll="118120"; __yadk_uid=AnCyvSq6ntlVHfdEhpnXJYbjysCDHU6L; _vwo_uuid_v2=DB0CC587D79CCED998D4110770B91D456|41eec305e761d862e477ef47612dede5; viewed="33443187"; gr_user_id=6e35b19f-e23c-4fb6-842a-18a20c980e17; bid=FWWyoPS44Ro; Hm_lvt_16a14f3002af32bf3a75dfe352478639=1641870927; __utmv=30149280.25272; __gads=ID=99f3ae0575c096ba-22f4369ec1d00069:T=1645853022:RT=1645853022:S=ALNI_Mb2wjGSlDECJiEVt-BsBkQUh7t3kQ; __utmc=30149280; __utmc=223695111; __utmz=223695111.1647143347.12.7.utmcsr=cnblogs.com|utmccn=(referral)|utmcmd=referral|utmcct=/; push_noty_num=0; push_doumail_num=0; ap_v=0,6.0; __utmz=30149280.1647163607.22.13.utmcsr=movie.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/subject/35613853/; _pk_ref.100001.4cf6=["","",1647167337,"https://www.cnblogs.com/"]; _pk_ses.100001.4cf6=*; __utma=30149280.9806854.1608881160.1647163607.1647167337.23; __utma=223695111.1031928723.1619963567.1647163842.1647167337.17; __utmb=223695111.0.10.1647167337; __utmt=1; __utmb=30149280.6.10.1647167337; dbcl2="252725314:EdgiL5WSU5A"; ck=q-_M; _pk_id.100001.4cf6=06112351ddd0e997.1619963567.17.1647168554.1647163974.'}
    header = {'Content-Type': 'text/html; charset=utf-8',
              'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.39'}  # 模拟浏览器
    r = requests.get(com_url, headers=header, cookies=Cookie, allow_redirects=False)
    # print(r.text)
    bs = BeautifulSoup(r.text, "lxml")
    print("ccc")
    # 获取并解析影评页

    # 取得元素，存入数组
    name = [i.find("a").text for i in bs.find_all("span", "comment-info")]  #用户名
    com_time = [i.text.strip() for i in bs.find_all("span", "comment-time")] #评论时间
    liking = [i.text for i in bs.find_all("span", "votes")] #有用数
    reply = [i.text for i in bs.find_all("span", "short")] #评论内容

    print(len(name),len(com_time),len(liking),len(reply))

    # 这里把每个获取到的元素成组放进数据库
    for j in range(19):
        value = (movie_name[i], name[j], com_time[j], liking[j], reply[j])
        insert_cominfo(value)

# 插入数据进入数据库
def insert_cominfo(value):
    # 连接数据库
    db = pymysql.Connect(host='localhost', user='root', password='123456', port=3306, db='movie')
    cursor = db.cursor()
    sql = "insert into replied_info(moviename, name, com_time, liking, reply) values(%s, %s, %s, %s, %s)"

    try:
        cursor.execute(sql, value)
        db.commit()
        print('插入数据成功')
    except Exception as e:
        db.rollback()
        print("执行MySQL: %s 时出错：%s" % (sql, e))
    db.close()

for i in range(49):
    print("开始爬取第{}个电影".format(i+1))
    for j in range(11):
        print("开始爬取第{}页".format(j+1))
        com_url = get_com_url(movie_urls[i], j)
        get_com(com_url, i)
        time.sleep(3)


